#coding=utf-8

'''
Created on 2011-9-22

@author: Administrator
'''
import sys
import datetime
import os
sys.path.append('/harvester/ripper/source/src/')

from ripper.core.Utils import getId #@UnresolvedImport
from ripper.handler.TorrentDownload import canDownload
from ripper.handler.Models import Forum, Post, Image, PostContent, Torrent, Url,RawContent #@UnresolvedImport
from ripper.core.Storage import db


class __SyncObj(object):

    def __init__(self):
        '''
        Constructor
        '''
        self.syncID = getId() # 同步ID 本次同步
    
    def transfer(self):
        forums = Forum.select()
        datas = self.query_data()
        count = datas.count()
        cur = 1
        for data in datas:
            print 'transfering ', cur, '/', count
            ret = self.save_data(data, forums)
            
            if False == ret : # 未下载完成
                print 'skipped.'
                continue
            
            elif None == ret: # 作废
                data['syncFlag'] = '2'
                data['syncDate'] = datetime.datetime.utcnow()
                db.save_item_data('data_sis', data)
                print 'discarded.'
                
            else : # 已同步
                data = ret
                data['syncFlag'] = '1'
                data['syncDate'] = datetime.datetime.utcnow()
                data['syncID'] = self.syncID
                db.save_item_data('data_sis', data)
                cur = cur + 1
        
    def query_data(self):
        q = db.db['data_sis'].find \
            ({'$or' : [{'syncFlag':None}, {'syncFlag' : '2'}] })    
        return q
    
    def save_data(self, data, forums):
        forum = data['forumId']
        
        if  data.has_key('ccontent') and \
            data.has_key('urls') and \
            data.has_key('images') and \
            data.has_key('torrents') : 
            
                
            # todo, 判断fail的IOError, 
            
            hasAtLeastOneImgDownloaded = False # 起码有一个下完
            allImagFaild = True
            for img in data['images']:
                
                if img['length'] == 0:
                    if img.has_key('fails') == False or img['fails'] == 0 :
                        return False # 未下载且无出错, 略过继续下一个
                else:
                    
                    # 失败超过3次不计入有效下载
                    if img['fails'] >=3:
                        continue
                        
                    # 判断是否垃圾文件
                    if img['length'] != 10240 and \
                         img['file'] != '404.jpg': # 404.jpg的文件长度
                        
                        if '.gif' not in img['file']: # 签名图...
                            
                            hasAtLeastOneImgDownloaded = True
                        
                    else:# 下载成功的
                        
                        allImagFaild = False
                    
            # 如果至少有一个下载完，其他都是fail, 则可以同步
            # 如果独苗图片是404或者小于10k则枪毙        
            if allImagFaild == True and hasAtLeastOneImgDownloaded == False:
                return None # 作废        
            
            
            hashedUrl = hash(data['postDetailUrl'])
            post = Post(forum=forum,pageUrl=data['postDetailUrl'],\
                    title=data['title'],titleSimple=data['titleSimple'],author=data['author'],postDate=data['postDate'],urlHash=hashedUrl)
            
            
            print 'postContent...'
            PostContent(post=post, content=data['ccontent'])
            print 'RawContent...'
            RawContent(post=post, content=data['rawContent'])
            
            print 'url...'
            for url in data['urls']:
                Url(post=post,name=url)
            
            print 'Image...'    
            for img in data['images']:
                Image(post=post,filename=img['file'],url=img['url'], fileLength=img['length'], downloaded=1)
            
            print 'Torrent...'    
            for torrent in data['torrents']:
                Torrent(post=post,filename=torrent['file'],url=torrent['url'],fileLength=torrent['length'], download=1)
            
            print data['postDetailUrl']
            data['syncTarget'] = post.id
            return data
        
        return False
    
    # 修复数据
    # 如果图片已经被(误)删除,则修改同步状态, 修改图片字段 
    def fix_data_if_img_not_existed(self):
        print 'fixing...'
        q = db.db['data_aisex'].find()  
        num = q.count()
        cur = 1
        for data in q:
            print 'checking...', cur, '/', num
            if data.has_key('syncFlag') and data['syncFlag'] == '1': # 已同步的数据
                # 检查图片是否存在, 否则取消同步状态
                if self.check_image_exists(data) == False:
                    print 'cancel syncFlag, reset image field...'
                    data['syncFlag'] = None
                    db.save_item_data('data_aisex', data)
                    cur = cur + 1
                else:
                    continue    
            
            else: # 未同步的数据
                
                if self.check_image_exists(data) == False:
                    print 'reset image field...'
                    cur = cur + 1
                else:
                    continue
                
    
    # 检查图片是否存在, 否则取消同步状态            
    def check_image_exists(self, data):
        if data.has_key('images') == False or data['images'] == []:
            return True
        ret = True
        for i in range(0, len(data['images'])):
            image = data['images'][i]
            
            if image['file'] != '' and image['length'] > 0L:
                f1 = os.path.join('/resource/images_aisex/', image['file'])
                f2 = os.path.join('/resource/images_aisex_pic/', image['file'])
                
                # 遍历图片
                if os.path.exists(f1) or os.path.exists(f2): # 存在, 继续遍历
                    continue
                
                else:# 存在误删除的图片
                    ret = False
                    image['length'] = 0L
                    image['file'] = ''
                    data['images'][i] = image
        
        if ret == False:
            db.save_item_data('data_aisex', data)
                
        return ret
    
    
def transfer_data():
    obj = __SyncObj()
    obj.transfer()
    
if __name__ == '__main__':
    obj = __SyncObj()
#    obj.fix_data_if_img_not_existed()
    obj.transfer()
    